{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c355a487",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/randy/workspace/runner/miniconda3/envs/ai-common/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "40b74033",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>距离川沙公路较近,但是公交指示不对,如果是\"蔡陆线\"的话,会非常麻烦.建议用别的路线.房间较...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>早餐太差，无论去多少人，那边也不加食品的。酒店应该重视一下这个问题了。房间本身很好。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>宾馆在小街道上，不大好找，但还好北京热心同胞很多~宾馆设施跟介绍的差不多，房间很小，确实挺小...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>CBD中心,周围没什么店铺,说5星有点勉强.不知道为什么卫生间没有电吹风</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7761</th>\n",
       "      <td>0</td>\n",
       "      <td>尼斯酒店的几大特点：噪音大、环境差、配置低、服务效率低。如：1、隔壁歌厅的声音闹至午夜3点许...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7762</th>\n",
       "      <td>0</td>\n",
       "      <td>盐城来了很多次，第一次住盐阜宾馆，我的确很失望整个墙壁黑咕隆咚的，好像被烟熏过一样家具非常的...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7763</th>\n",
       "      <td>0</td>\n",
       "      <td>看照片觉得还挺不错的，又是4星级的，但入住以后除了后悔没有别的，房间挺大但空空的，早餐是有但...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7764</th>\n",
       "      <td>0</td>\n",
       "      <td>我们去盐城的时候那里的最低气温只有4度，晚上冷得要死，居然还不开空调，投诉到酒店客房部，得到...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7765</th>\n",
       "      <td>0</td>\n",
       "      <td>说实在的我很失望，之前看了其他人的点评后觉得还可以才去的，结果让我们大跌眼镜。我想这家酒店以...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7765 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                             review\n",
       "0         1  距离川沙公路较近,但是公交指示不对,如果是\"蔡陆线\"的话,会非常麻烦.建议用别的路线.房间较...\n",
       "1         1                       商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!\n",
       "2         1         早餐太差，无论去多少人，那边也不加食品的。酒店应该重视一下这个问题了。房间本身很好。\n",
       "3         1  宾馆在小街道上，不大好找，但还好北京热心同胞很多~宾馆设施跟介绍的差不多，房间很小，确实挺小...\n",
       "4         1               CBD中心,周围没什么店铺,说5星有点勉强.不知道为什么卫生间没有电吹风\n",
       "...     ...                                                ...\n",
       "7761      0  尼斯酒店的几大特点：噪音大、环境差、配置低、服务效率低。如：1、隔壁歌厅的声音闹至午夜3点许...\n",
       "7762      0  盐城来了很多次，第一次住盐阜宾馆，我的确很失望整个墙壁黑咕隆咚的，好像被烟熏过一样家具非常的...\n",
       "7763      0  看照片觉得还挺不错的，又是4星级的，但入住以后除了后悔没有别的，房间挺大但空空的，早餐是有但...\n",
       "7764      0  我们去盐城的时候那里的最低气温只有4度，晚上冷得要死，居然还不开空调，投诉到酒店客房部，得到...\n",
       "7765      0  说实在的我很失望，之前看了其他人的点评后觉得还可以才去的，结果让我们大跌眼镜。我想这家酒店以...\n",
       "\n",
       "[7765 rows x 2 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('/Users/randy/workspace/project/opensource/llm-learn/transformers-code/01-Getting Started/04-model/ChnSentiCorp_htl_all.csv')\n",
    "data.head(3)\n",
    "data = data.dropna()\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8c8391da",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset\n",
    "\n",
    "class MyDataset(Dataset):\n",
    "  def __init__(self, csv_path:str) -> None:\n",
    "    super().__init__()\n",
    "    self.data = pd.read_csv(csv_path)\n",
    "    self.data = self.data.dropna()\n",
    "\n",
    "  def __getitem__(self, index):\n",
    "    row = self.data.iloc[index]\n",
    "    return row['review'], row['label']\n",
    "  \n",
    "  def __len__(self):\n",
    "    return len(self.data)\n",
    "\n",
    "dataset = MyDataset('/Users/randy/workspace/project/opensource/llm-learn/transformers-code/01-Getting Started/04-model/ChnSentiCorp_htl_all.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "979a22a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('距离川沙公路较近,但是公交指示不对,如果是\"蔡陆线\"的话,会非常麻烦.建议用别的路线.房间较为简单.', 1)\n",
      "('商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!', 1)\n",
      "('早餐太差，无论去多少人，那边也不加食品的。酒店应该重视一下这个问题了。房间本身很好。', 1)\n",
      "('宾馆在小街道上，不大好找，但还好北京热心同胞很多~宾馆设施跟介绍的差不多，房间很小，确实挺小，但加上低价位因素，还是无超所值的；环境不错，就在小胡同内，安静整洁，暖气好足-_-||。。。呵还有一大优势就是从宾馆出发，步行不到十分钟就可以到梅兰芳故居等等，京味小胡同，北海距离好近呢。总之，不错。推荐给节约消费的自助游朋友~比较划算，附近特色小吃很多~', 1)\n",
      "('CBD中心,周围没什么店铺,说5星有点勉强.不知道为什么卫生间没有电吹风', 1)\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "  print(dataset[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "13968123",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6212, 1553)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torch.utils.data import random_split\n",
    "\n",
    "train, valid = random_split(dataset, lengths=[0.8, 0.2])\n",
    "len(train), len(valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "080149cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "tokinizer = AutoTokenizer.from_pretrained(\"hfl/rbt3\")\n",
    "\n",
    "def collate_fn(batch):\n",
    "  texts, labels = [],[]\n",
    "  for item in batch:\n",
    "    texts.append(item[0])\n",
    "    labels.append(item[1])\n",
    "  \n",
    "  inputs = tokinizer(texts, max_length=128, padding=\"max_length\", truncation=True, return_tensors='pt')\n",
    "  inputs['labels'] = torch.tensor(labels)\n",
    "  return inputs\n",
    "\n",
    "train_loader = DataLoader(train, shuffle=True, batch_size=12, collate_fn=collate_fn)\n",
    "valid_loader = DataLoader(valid, shuffle=False, batch_size=32, collate_fn=collate_fn)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1f8c2ab",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "15429805",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[ 101, 3221, 2769,  ..., 5892, 1355,  102],\n",
       "        [ 101, 4677, 3819,  ...,    0,    0,    0],\n",
       "        [ 101, 6370, 4638,  ...,  671, 7313,  102],\n",
       "        ...,\n",
       "        [ 101, 6432, 1368,  ...,  678, 1343,  102],\n",
       "        [ 101, 2899,  758,  ..., 6821, 3416,  102],\n",
       "        [ 101, 2791, 7313,  ...,    0,    0,    0]]), 'token_type_ids': tensor([[0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1],\n",
       "        [1, 1, 1,  ..., 0, 0, 0],\n",
       "        [1, 1, 1,  ..., 1, 1, 1],\n",
       "        ...,\n",
       "        [1, 1, 1,  ..., 1, 1, 1],\n",
       "        [1, 1, 1,  ..., 1, 1, 1],\n",
       "        [1, 1, 1,  ..., 0, 0, 0]]), 'labels': tensor([0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1])}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(iter(train_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0f39a5c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at hfl/rbt3 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "is_gpu = torch.mps.is_available()\n",
    "device = torch.device(\"mps\") if torch.backends.mps.is_available() else torch.device(\"cpu\")\n",
    "model = AutoModelForSequenceClassification.from_pretrained('hfl/rbt3')\n",
    "if is_gpu:\n",
    "  model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "dc2abb6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.optim import Adam\n",
    "optimizer = Adam(model.parameters(), lr = 2e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2c106ff1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate():\n",
    "  model.eval()\n",
    "  acc_num = 0\n",
    "  with torch.inference_mode():\n",
    "    for batch in valid_loader:\n",
    "      if is_gpu:\n",
    "        batch = {k: v.to(device)for k,v in batch.items()}\n",
    "      outputs = model(**batch)\n",
    "      pred = torch.argmax(outputs.logits, dim=-1)\n",
    "      acc_num += (pred.long() == batch['labels'].long()).float().sum()\n",
    "  return acc_num / len(valid)\n",
    "\n",
    "def train(eporch=5, log_step=100):\n",
    "  global_step = 0\n",
    "  for ep in range(eporch):\n",
    "    model.train()\n",
    "    for batch in train_loader:\n",
    "      if torch.mps.is_available():\n",
    "        batch = {k:v.to(device) for k,v in batch.items()}\n",
    "      optimizer.zero_grad()\n",
    "      output = model(**batch)\n",
    "      output.loss.backward()\n",
    "      optimizer.step()\n",
    "      if global_step % log_step == 0:\n",
    "        print(f'ep:{ep}, global_step:{global_step}, loss:{output.loss.item()}')\n",
    "      global_step += 1\n",
    "    acc = evaluate()\n",
    "    print(f\"ep: {ep}, acc: {acc}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "747033f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep:0, global_step:0, loss:0.002075817435979843\n",
      "ep:0, global_step:100, loss:0.1671174168586731\n",
      "ep:0, global_step:200, loss:0.0016467766836285591\n",
      "ep:0, global_step:300, loss:0.04035833105444908\n",
      "ep:0, global_step:400, loss:0.08247564733028412\n",
      "ep:0, global_step:500, loss:0.02048506774008274\n",
      "ep: 0, acc: 0.8898905515670776\n",
      "ep:1, global_step:600, loss:0.07115166634321213\n",
      "ep:1, global_step:700, loss:0.001541023957543075\n",
      "ep:1, global_step:800, loss:0.00994240865111351\n",
      "ep:1, global_step:900, loss:0.0011474695056676865\n",
      "ep:1, global_step:1000, loss:0.00902046263217926\n",
      "ep: 1, acc: 0.8892466425895691\n",
      "ep:2, global_step:1100, loss:0.0013278829865157604\n",
      "ep:2, global_step:1200, loss:0.002372784772887826\n",
      "ep:2, global_step:1300, loss:0.0074618360958993435\n",
      "ep:2, global_step:1400, loss:0.012253456749022007\n",
      "ep:2, global_step:1500, loss:0.09168172627687454\n",
      "ep: 2, acc: 0.8924661874771118\n",
      "ep:3, global_step:1600, loss:0.0020350029226392508\n",
      "ep:3, global_step:1700, loss:0.0005392524180933833\n",
      "ep:3, global_step:1800, loss:0.002019723644480109\n",
      "ep:3, global_step:1900, loss:0.0028346723411232233\n",
      "ep:3, global_step:2000, loss:0.005685294046998024\n",
      "ep: 3, acc: 0.8943979144096375\n",
      "ep:4, global_step:2100, loss:0.024413013830780983\n",
      "ep:4, global_step:2200, loss:0.013039175420999527\n",
      "ep:4, global_step:2300, loss:0.0004548093711491674\n",
      "ep:4, global_step:2400, loss:0.001771321869455278\n",
      "ep:4, global_step:2500, loss:0.0025186145212501287\n",
      "ep: 4, acc: 0.8789439797401428\n"
     ]
    }
   ],
   "source": [
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "25186f08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "好评！\n"
     ]
    }
   ],
   "source": [
    "sen = \"酒店不错，菜品也挺好吃\"\n",
    "model.eval()\n",
    "id2_label = {0: \"差评！\", 1: \"好评！\"}\n",
    "with torch.inference_mode():\n",
    "  inputs = tokinizer(sen, max_length=128, padding=\"max_length\", truncation=True, return_tensors='pt')\n",
    "  inputs = {k:v.to(device) for k,v in inputs.items()}\n",
    "  outputs = model(**inputs)\n",
    "  pred = torch.argmax(outputs.logits, dim=-1)\n",
    "  print(id2_label.get(pred.item()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "0efdc7d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Device set to use mps\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "model.config.id2label = id2_label\n",
    "pl = pipeline('text-classification', model = model, tokenizer=tokinizer, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "32616664",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': '好评！', 'score': 0.9941821694374084}]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl(sen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ai-common",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
